from __future__ import with_statement

#from xml.etree.ElementTree import ElementTree
#import time

from Kaleidoscope.KSColumn import KSColumn
from Kaleidoscope.KSColumnBSDDBDuplicate import KSColumnBSDDBDuplicate
from Kaleidoscope.KSColumnBSDDBUnique import KSColumnBSDDBUnique
#from Kaleidoscope.KSQuery import KSQuery
from Kaleidoscope.KSTable import KSTable
from Kaleidoscope.KSColumnPlusIndex import KSColumnPlusIndex
from Kaleidoscope.KSUtility import KSUtility
from Kaleidoscope.KSEventManager import KSEventManager
import twitter

from bsddb import db
import mx.DateTime
import uuid
import sys
from lxml import etree
import re
from pdis.xpath import compile
import ply.yacc as yacc
import ply.lex as lex
import time
from urllib2 import HTTPError


class KSTwitterbaseError(Exception):
    """
    Exception class for KSSystem

    """
    B_UNKNOWN_EXCEPTION = True
    B_CLASS_EXCEPTION = False


    def __init__(self, u_message):
        self.__u_message = u_message
    def __str__(self):
        return repr(self.__u_message)

    @staticmethod
    def print_and_raise_error(u_message, b_unknown_exception):
        """
        Convenience method for printing and raising an error for this
        particular Exception subclass.

        """
        u_message = u'KSTwitterbase.py: ' + u_message
        sys.stderr.write(u_message + u'\n')

        if (b_unknown_exception):
            raise Exception(u_message)
        else:
            raise KSTwitterbaseError(u_message)


#
# Exception classes
#
class LexerError(Exception):
    def __init__(self, msg, lineno, columnno, twitterapi, user):
        self.msg = msg
        self.lineno = lineno
        self.columnno = columnno
        self.l_twitterapi = twitterapi
        self.l_user = user

    def show(self):
        u_output_text = u'Lexer error (%d, %d) %s\n' % \
            (self.lineno, self.columnno, self.msg)
        sys.stderr.write(u_output_text)
        self.l_twitterapi.PostDirectMessage(
            self.l_user, u_output_text)


class ParserError(Exception):
    def __init__(self, msg, lineno, columnno, twitterapi, user):
        self.msg = msg
        self.lineno = lineno
        self.columnno = columnno
        self.p_twitterapi = twitterapi
        self.p_user = user

    def show(self):
        u_message = u'Parser error (%d, %d) %s\n' % \
            (self.lineno, self.columnno, self.msg)
        sys.stderr.write(u_message)
        self.p_twitterapi.PostDirectMessage(
            self.p_user, u_message)


class KSTwitterbase(object):
    """
    This class will provide the database operations and the command processing.

    """
    # Security flags
    B_SECURITY__PRIVATE = True
    B_SECURITY__PUBLIC = False


    #
    # Lexer specification
    #
    tokens = (
        'UNICODE', 'IDENTIFIER', 'PUBLIC', 'PRIVATE', 'INT_NUMBER',
        'FLOAT_NUMBER')

    # Tokens
    t_PUBLIC = r'\[\]\.'
    t_PRIVATE = r'\]\[\.'
    t_IDENTIFIER =  r'[_a-zA-Z][_a-zA-Z0-9]*'
    #t_HALT = r'\!\#WAD2qse\~\*\)OK\:9ilp\+'
    #t_INPUT = r'<<'    # <<!
    #t_DELETE = r'\/\/'
    #t_TRIGGER = r'\*\*'
    #t_OUTPUT = r'>>'
    #t_GREATER_THAN_EQUAL = r'>\='
    #t_LESS_THAN_EQUAL = r'<\='
    #t_EQUAL = r'\=\='
    #t_NOT_EQUAL = r'\!\='

    literals = ['@', '#', '.', ':', '=', '?']

    # Ignore whitespace
    t_ignore = ' \t'


    def t_FLOAT_NUMBER(self, token):
        r'\-?\d+\.(\d+)?'
        try:
            token.value = float(token.value)

        except ValueError:
            u_message = u'Value error: decimal number %s is too large. ' \
                + u'Setting value to 0.0 .' % token.value
            print u_message
            self.twitterapi.PostDirectMessage(
                self.u_cmd_username, u_message)
            print
            token.value = 0.0

        return token


    def t_INT_NUMBER(self, token):
        r'\-?\d+'
        try:
            token.value = int(token.value)

        except ValueError:
            u_message = u'Value error: integer number %s is too large. ' \
                + u'Setting value to 0.' % token.value
            print u_message
            self.twitterapi.PostDirectMessage(
                self.u_cmd_username, u_message)
            print

            token.value = 0

        return token


    def t_UNICODE(self, token):
        # If there is time, besides doing a rough translation of word to id,
        # a grammatical parse of the words (where applicable) could be returned.
        r' \".*\" '
        try:
            with self.ksutil.transaction(self.__kstx_transaction,
                self.t_UNICODE) as kstx_transaction:

                # Remove the surrounding quotes at the start and end
                token.value = unicode(token.value[1:-1])

        except ValueError:
            u_message = u'String error: the string of characters %s was not ' \
                + u'formatted correctly. Remember to use "".' % token.value
            print u_message
            self.twitterapi.PostDirectMessage(
                self.u_cmd_username, u_message)
            print

            token.value = None

        return token


    def t_newline(self, token):
        r'\n+|\r+'
        #r'\n+'
        token.lineno += token.value.count("\n")


    def t_error(self, token):
        msg = "Illegal character '%s'" % (token.value[0])
        #self.twitterapi.PostDirectMessage(self.u_cmd_username, msg)
        raise LexerError(msg, token.lineno, token.lexer.lexpos,
                         self.twitterapi, self.u_cmd_username)

    #
    # Parser specification
    #
    """
    GRAMMAR:

    start_section: delete_command
        |          assign_command
        |          xpath_command
        |          list_command
        |          directory_command
        |          halt_command

    scoped_identifier: PUBLIC identifier_list
        |              PRIVATE identifier_list

    identifier_list: identifier_list '.' IDENTIFIER
        |            IDENTIFIER

    delete_command: '#' scoped_identifier

    assign_command: scoped_identifier '=' UNICODE

    xpath_command: '@' scoped_identifier ':' scoped_identifier

    directory_command: '?' scoped_identifier

    list_command: scoped_identifier

    halt_command: !#WAD2qse~*)OK:9ilp

    """

    def p_start_section_1(self, token):
        "start_section : delete_command"
        token[0] = token[1]

    def p_start_section_2(self, token):
        "start_section : assign_command"
        token[0] = token[1]

    def p_start_section_3(self, token):
        "start_section : xpath_command"
        print token[1]
        token[0] = token[1]

    def p_start_section_5(self, token):
        "start_section : list_command"
        print token[1]
        token[0] = token[1]

    '''
    def p_start_section_7(self, token):
        "start_section : halt_command"
        print token[1]
        token[0] = token[1]
    '''

    def p_start_section_6(self, token):
        "start_section : directory_command"
        print token[1]
        token[0] = token[1]

    '''
    def p_halt_command_1(self, token):
        "halt_command : HALT"
        token[0]=u'!@#$%^&*()(*&^%$#@!'
    '''

    def p_scoped_identifier_1(self, token):
        "scoped_identifier : PUBLIC identifier_list"
        # Format: o.username.whatever
        i_identifier_count = len(token[2])
        if (i_identifier_count >= 1):
            token[0] = [KSTwitterbase.B_SECURITY__PUBLIC, token[2]]
        else:
            token[0] = u'Identifier error: public data format -> ' \
                + u'[].username.whatever.'
            KSTwitterbaseError.print_and_raise_error(u'The identifier list '
                + u'must include at least the username and one more identifier.',
                KSTwitterbaseError.B_UNKNOWN_EXCEPTION)


    def p_scoped_identifier_2(self, token):
        "scoped_identifier : PRIVATE identifier_list"
        # Format: x.username.whatever
        # Identifier list is purely under the control of the user; no
        # username should appear here
        token[0] = [KSTwitterbase.B_SECURITY__PRIVATE, token[2], token[2][0]]


    def p_identifier_list_1(self, token):
        "identifier_list : identifier_list '.' IDENTIFIER"
        token[0] = []
        for a_token in token[1]:
            token[0].append(a_token)
        token[0].append(token[3])

    def p_identifier_list_2(self, token):
        "identifier_list : IDENTIFIER"
        token[0] = [token[1]]


    def p_delete_command_1(self, token):
        "delete_command : '#' scoped_identifier"
        b_security_scope = token[2][0]
        list_identifier = token[2][1]
        u_identifier = self.__assemble_dotted_identifier(list_identifier)

        with self.ksutil.transaction(None, self.p_delete_command_1) \
            as kstx_transaction:

            try:
                # Begin with the security issue
                if (b_security_scope == KSTwitterbase.B_SECURITY__PUBLIC):
                    kst_table_current = self.table_public
                    self.b_display_public_message = True
                else:
                    kst_table_current = self.table_private
                    self.b_display_public_message = False

                list_keys = kst_table_current.get_row_by_primary_value(
                    u_identifier, [u'username'], kstx_transaction)

                if (list_keys == None):
                    # Item does not exist
                    token[0] = u'Deletion failed: the item %s does not exist.' \
                        % u_identifier
                else:
                    i_key_count = len(list_keys)
                    if (i_key_count == 1):
                        # Data exists, figure out who owns it and
                        # if deletion is ok
                        u_item_owner = list_keys.pop()
                        if (u_item_owner == self.u_cmd_username):
                            # Current user owns the data
                            kst_table_current.delete_row_by_primary_value(
                                u_identifier, kstx_transaction)
                        else:
                            token[0] = u'Deletion failed: cannot delete data ' \
                                + u'that is not your own; data owner %s.' \
                                % u_item_owner
                    else:
                        token[0] = u'Deletion failed: duplicate identifiers ' \
                            + u'exist for %s.' % u_identifier
            except Exception:
                token[0] = u'Deletion failed: internal database error.'


    def p_assign_command_1(self, token):
        "assign_command : scoped_identifier '=' rhs_expr"
        # Handles the case where a new variable is created and assigned a value
        # or if an existing variable has its value changed.

        with self.ksutil.transaction(self.__kstx_transaction,
            self.p_assign_command_1) as kstx_transaction:

            b_security_scope = token[1][0]
            list_identifier = token[1][1]
            #u_identifier = token[1][1]
            u_variable_value = token[3]
            u_identifier_username = list_identifier[0]
            u_identifier = self.__assemble_dotted_identifier(list_identifier)

            try:
                # Begin with the security issue
                if (b_security_scope == KSTwitterbase.B_SECURITY__PUBLIC):
                    kst_table_current = self.table_public
                    ksc_primary_key = self.ksc_public_varname
                    self.b_display_public_message = True
                else:
                    kst_table_current = self.table_private
                    ksc_primary_key = self.ksc_private_varname
                    self.b_display_public_message = False

                list_keys = kst_table_current.get_row_by_primary_value(
                    u_identifier, [u'username'], kstx_transaction)

                if (list_keys == None):
                    # Data does not exist, ok only if first user part
                    # equals the username; if so, add the row
                    if (u_identifier_username == self.u_cmd_username):
                        obj_key = ksc_primary_key.generate_id()
                        kst_table_current.add_row(obj_key,
                            {u'varname': u_identifier,
                             u'varvalue': u_variable_value,
                             u'username': self.u_cmd_username},
                            kstx_transaction)
                    else:
                        token[0] = u'Update failed: Identifier begin with your ' \
                            + u'username "%s".' % self.u_cmd_username
                else:
                    i_key_count = len(list_keys)
                    if (i_key_count == 1):
                        # See if the user owns the data
                        u_item_owner = list_keys.pop()
                        if (u_item_owner == self.u_cmd_username):
                            list_keys = ksc_primary_key.get_keys_from_data_value(
                                u_identifier, kstx_transaction,
                                KSColumn.B_REGULAR_DATA)

                            i_key_count = len(list_keys)
                            if (i_key_count == 1):
                                obj_key = list_keys.pop()
                                kst_table_current.change_row(obj_key,
                                    {u'varname': u_identifier,
                                     u'varvalue': u_variable_value,
                                     u'username': self.u_cmd_username},
                                    kstx_transaction)
                            else:
                                token[0] = u'Update failed: duplicate ' \
                                    + 'identifiers exist for %s.' % u_identifier
                        else:
                            token[0] = u'Update failed: cannot update data ' \
                                + u'that is not your own; data owner %s.' \
                                % u_item_owner
                    else:
                        token[0] = u'Update failed: duplicate identifier. ' \
                            + u'exist for %s.' % u_identifier
            except Exception:
                token[0] = u'Update failed: internal database error.'


    def p_xpath_command_1(self, token):
        "xpath_command : '@' scoped_identifier ':' scoped_identifier"

        with self.ksutil.transaction(self.__kstx_transaction,
            self.p_xpath_command_1) as kstx_transaction:

            b_security_scope__url = token[2][0]
            list_identifier__url = token[2][1]
            b_security_scope__xpath = token[4][0]
            list_identifier__xpath = token[4][1]

            u_identifier__url = self.__assemble_dotted_identifier(
                list_identifier__url)
            u_identifier__xpath = self.__assemble_dotted_identifier(
                list_identifier__xpath)


            # Both parts have to be public for the command to be posted
            try:
                if (b_security_scope__url == KSTwitterbase.B_SECURITY__PUBLIC):
                    kst_table_url_current = self.table_public
                    self.b_display_public_message = True
                else:
                    kst_table_url_current = self.table_private
                    self.b_display_public_message = False

                if (b_security_scope__xpath == KSTwitterbase.B_SECURITY__PUBLIC):
                    kst_table_xpath_current = self.table_public
                    self.b_display_public_message &= True
                else:
                    kst_table_xpath_current = self.table_private
                    self.b_display_public_message &= False


                if ((self.__ok_to_read_data(kst_table_url_current,
                    u_identifier__url, b_security_scope__url, kstx_transaction))
                    and
                    (self.__ok_to_read_data(kst_table_url_current,
                    u_identifier__xpath, b_security_scope__xpath, kstx_transaction))):

                    # Get the URL
                    list_values = kst_table_url_current.\
                        get_row_by_primary_value(u_identifier__url,
                        [u'varvalue'], kstx_transaction)

                    if (list_values == None):
                        token[0]= u'URL/Xpath failed: there is no URL for %s.' \
                            % u_identifier__url
                    else:
                        i_row_count = len(list_values)

                        if (i_row_count == 1):
                            u_value__url = list_values.pop()
                        else:
                            token[0]= u'URL/Xpath failed: duplicate values ' \
                                + u'exist for %s.' % u_identifier__url

                        # Get the XPath
                        list_values = kst_table_xpath_current.\
                            get_row_by_primary_value(u_identifier__xpath,
                                [u'varvalue'], kstx_transaction)

                        if (list_values == None):
                            token[0]= u'URL/Xpath failed: there is no XPath ' \
                                + u'for %s.' % u_identifier__xpath
                        else:
                            i_row_count = len(list_values)
                            if (i_row_count == 1):
                                u_value__xpath = list_values.pop()
                            else:
                                token[0]= u'URL/Xpath failed: duplicate values ' \
                                    + u'exist for %s.' % u_identifier__xpath

                            token[0] = self.xpath_processing(
                                u_value__url, u_value__xpath)
                else:
                    token[0] = u'URL/Xpath failed: if any identifiers are ' \
                        + u'private, they must be your own.'

            except Exception:
                if (not isinstance(token[1], list)):
                    token[0] = token[1]
                else:
                    token[0] = token[3]
                KSTwitterbaseError.print_and_raise_error(u'There was an unknown '
                    + u'error while trying to retrieve the data item.',
                    KSTwitterbaseError.B_UNKNOWN_EXCEPTION)


    def p_list_command_1(self, token):
        "list_command : scoped_identifier"

        # Begin with the security issue
        with self.ksutil.transaction(self.__kstx_transaction,
            self.p_list_command_1) as kstx_transaction:

            b_security_scope = token[1][0]
            list_identifier = token[1][1]
            u_identifier = self.__assemble_dotted_identifier(list_identifier)

            try:
                if (b_security_scope == KSTwitterbase.B_SECURITY__PUBLIC):
                    kst_table_current = self.table_public
                    self.b_display_public_message = True
                else:
                    kst_table_current = self.table_private
                    self.b_display_public_message = False

                if (self.__ok_to_read_data(kst_table_current,
                    u_identifier, b_security_scope, kstx_transaction)):

                    list_of_lists = kst_table_current.get_row_by_primary_value(
                        u_identifier, [u'varvalue'], kstx_transaction)

                    if (list_of_lists == None):
                        token[0] = u'List failed: there is no value for %s.' \
                            % u_identifier
                    else:
                        i_row_count = len(list_of_lists)
                        if (i_row_count == 1):
                            token[0] = list_of_lists.pop()
                        else:
                            token[0] = u'List failed: duplicate values exist ' \
                                + u'for %s.' % u_identifier
                else:
                    token[0] = 'List failed: User %s cannot view the data of ' \
                        + u'%s.' % (self.u_cmd_username, u_identifier)

            except Exception:
                token[0] = token[1]


    def p_directory_command_1(self, token):
        "directory_command : '?' scoped_identifier"
        # List the items only one level down.
        # EX: [].username.stuff will list the items that have one more
        # namespace below stuff
        # Begin with the security issue
        with self.ksutil.transaction(self.__kstx_transaction,
            self.p_directory_command_1) as kstx_transaction:

            b_security_scope = token[2][0]
            list_identifier = token[2][1]
            i_identifier_partcount = len(list_identifier)
            #u_identifier = self.__assemble_dotted_identifier(list_identifier)

            try:
                if (b_security_scope == KSTwitterbase.B_SECURITY__PUBLIC):
                    kst_table_current = self.table_public
                    ksc_varname = self.ksc_public_varname
                    ksc_username = self.ksc_public_username

                    # Public...all items are open
                    fn_ok_to_include_data = lambda x: True

                else:
                    kst_table_current = self.table_private
                    ksc_varname = self.ksc_private_varname
                    ksc_username = self.ksc_private_username

                    # Private...only show items owned by the user
                    fn_ok_to_include_data = lambda x: \
                        self.u_cmd_username == x

                # First, get all of the keys owned by the user
                # Get an identifier, split by ., count the length.
                # If length of identifier is one more than the length
                # of the search, get the item name, and add one to its
                # dictionary entry.
                dict_item_count = {}

                # This is a bad idea, but it will have to do for now
                list_all_keys = kst_table_current.keys(kstx_transaction)

                for obj_key in list_all_keys:
                    list_data = kst_table_current.get_row(
                        obj_key, [u'varname', u'username'], kstx_transaction)

                    if (len(list_data) != 2):
                        token[0] = u'Directory errror: Unable to retrieve ' \
                            + u'rows of data from the table.'
                        break
                    else:
                        list_varname = list_data[0].split(u'.')
                        i_current_identifier_parts = len(list_varname)
                        u_username = list_data[1]

                        # If the current identifier has fewer parts than the
                        # identifier to query, skip it
                        if (i_current_identifier_parts < i_identifier_partcount):
                            continue

                        if (fn_ok_to_include_data(list_varname[0])):
                            # Analyze each part
                            b_match = True
                            for i_index in xrange(0, i_identifier_partcount):
                                b_match &= (list_identifier[i_index] ==
                                    list_varname[i_index])

                            if (b_match):
                                if (i_current_identifier_parts ==
                                    i_identifier_partcount):
                                    # Should only occur once...this is where
                                    # the queried identifier actually has
                                    # its own value besides identifiers
                                    # below it
                                    u_varname = u'*'
                                else:
                                    u_varname = \
                                        list_varname[i_identifier_partcount]

                                if (dict_item_count.has_key(u_varname)):
                                    dict_item_count[u_varname] = \
                                        dict_item_count[u_varname] + 1
                                else:
                                    dict_item_count[u_varname] = 1
                            # else, do not include the item in the list
                        # else, skip this data item
                    # end if
                # end of loop

                # Now assemble the output
                u_output = u''
                if (len(dict_item_count) == 0):
                    u_output = u'No entries.'
                else:
                    for u_itemname in dict_item_count:
                        u_new_part = u'%s=%d,' % (u_itemname,
                            dict_item_count[u_itemname])
                        u_output += u_new_part
                    u_output = u_output[:-1]

                token[0] = u_output

            except Exception:
                token[0] = token[1]
                KSTwitterbaseError.print_and_raise_error(u'There was an unknown '
                    + u'error while trying to retrieve the data item.',
                    KSTwitterbaseError.B_UNKNOWN_EXCEPTION)


    def p_rhs_expr_1(self, token):
        "rhs_expr : UNICODE"
        token[0] = token[1]

    def p_rhs_expr_2(self, token):
        "rhs_expr : FLOAT_NUMBER"
        token[0] = unicode(token[1])

    def p_rhs_expr_3(self, token):
        "rhs_expr : INT_NUMBER"
        token[0] = unicode(token[1])

    def p_rhs_expr_4(self, token):
        "rhs_expr : list_command"
        token[0] = token[1]



    def p_error(self, token):
        try:
            msg = "Syntax error at '%s'" % token.value
            raise ParserError(msg, token.lineno, token.lexer.lexpos,
                              self.twitterapi, self.u_cmd_username)
        except AttributeError:
            pass


    def __assemble_dotted_identifier(self, list_identifier):
        """
        Used to created a unique dotted identifier for the variable.

        """
        u_identifier = list_identifier[0]

        for i_index in xrange(1, len(list_identifier)):
            u_identifier = u_identifier + u'.' + list_identifier[i_index]

        return u_identifier


    def __ok_to_read_data(self, table_current, u_identifier, b_security_scope,
                              kstx_transaction):
        """
        Any user can read public identifiers. Only the owner can read private
        data.

        """
        try:
            list_keys = table_current.get_row_by_primary_value(
                u_identifier, [u'username'], kstx_transaction)

            if (list_keys == None):
                # Item does not exist...but it is ok to read nothing
                return True
                #KSTwitterbaseError.print_and_raise_error(u'The item '
                #    + u'%s does not exist.' % u_identifier,
                #    KSTwitterbaseError.B_UNKNOWN_EXCEPTION)
            else:
                i_key_count = len(list_keys)
                if (i_key_count == 1):
                    # See if the user owns the data
                    u_item_owner = list_keys.pop()
                    if (u_item_owner == self.u_cmd_username):
                        return True
                    else:
                        if (b_security_scope == KSTwitterbase.B_SECURITY__PUBLIC):
                            return True
                        else:
                            return False
                else:
                    KSTwitterbaseError.print_and_raise_error(u'Duplicate '
                        + u'variables exist in the database.',
                        KSTwitterbaseError.B_CLASS_EXCEPTION)

        except Exception:
            KSTwitterbaseError.print_and_raise_error(u'There was an unknown '
                + u'error while trying to verify if the user can read the data.',
                KSTwitterbaseError.B_UNKNOWN_EXCEPTION)


    def __init__(self):
        self.prompt = '1' # startup value
        self.uuid_number = uuid.uuid4()
        self.parser = etree.HTMLParser()
        self.u_cmd_username = u''
        self.b_display_public_message = True
        self.__kstx_transaction = None

        self.twitterapi = twitter.Api(username='', password='')

        # ENVIRONMENT
        print 'starting...'
        self.i_flags = db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK \
            | db.DB_INIT_LOG | db.DB_INIT_TXN #| db.DB_PRIVATE

        self.env = db.DBEnv(0)
        self.env.set_cachesize(0, 5000000, 5)
        self.env.open('C:\\data_kaleidoscope', self.i_flags, 0)
        print 'created environment'

        try:
            # Do a checkpoint if necessary
            self.env.txn_checkpoint(db.DB_FORCE, 1000, 0)
        except Exception:
            print 'better luck next time'

        # Two tables...one for url, one for xpath
        self.ksc_public_varname = KSColumnBSDDBUnique(u'public_varname.db', 0,
            mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_UNIQUE_VALUES)

        self.ksc_public_varvalue = KSColumnBSDDBUnique(u'public_varvalue.db', 0,
            mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_DUPLICATE_VALUES)

        self.ksc_public_username = KSColumnBSDDBUnique(u'public_username.db', 0,
            mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_DUPLICATE_VALUES)
        self.ksc_public_username_index = KSColumnBSDDBDuplicate(
            u'public_username_index.db', 0, mx.DateTime.Date(2008, 9, 10),
            self.env, u'unicode', KSColumn.B_PICKLE_KEY)
        self.kscpi_public_username = KSColumnPlusIndex(
            self.ksc_public_username, self.ksc_public_username_index)

        self.table_public = KSTable(u'public')
        self.table_public.add_column(u'varname', self.ksc_public_varname)
        self.table_public.add_column(u'varvalue', self.ksc_public_varvalue)
        self.table_public.add_column(u'username', self.kscpi_public_username)
        self.table_public.set_primary_column(u'varname')


        self.ksc_private_varname = KSColumnBSDDBUnique(u'private_varname.db', 0,
            mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_UNIQUE_VALUES)

        self.ksc_private_varvalue = KSColumnBSDDBUnique(u'private_varvalue.db',
            0, mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_DUPLICATE_VALUES)

        self.ksc_private_username = KSColumnBSDDBUnique(u'private_username.db',
            0, mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_DUPLICATE_VALUES)
        self.ksc_private_username_index = KSColumnBSDDBDuplicate(
            u'private_userid_index.db', 0, mx.DateTime.Date(2008, 9, 10),
            self.env, u'unicode', KSColumn.B_PICKLE_KEY)
        self.kscpi_private_username = KSColumnPlusIndex(
            self.ksc_private_username, self.ksc_private_username_index)

        self.table_private = KSTable(u'private')
        self.table_private.add_column(u'varname', self.ksc_private_varname)
        self.table_private.add_column(u'varvalue', self.ksc_private_varvalue)
        self.table_private.add_column(u'username', self.kscpi_private_username)
        self.table_private.set_primary_column(u'varname')


        self.ksc_queue_msgid = KSColumnBSDDBUnique(u'queue_msgid.db', 0,
            mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_UNIQUE_VALUES)

        self.ksc_queue_username = KSColumnBSDDBUnique(u'queue_username.db', 0,
            mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_DUPLICATE_VALUES)

        self.ksc_queue_msgtext = KSColumnBSDDBUnique(u'queue_msgtext.db', 0,
            mx.DateTime.Date(2008, 9, 10), self.env, u'int',
            KSColumn.B_PACK_KEY, KSColumn.B_DUPLICATE_VALUES)

        self.table_queue = KSTable(u'queue')
        self.table_queue.add_column(u'msgid', self.ksc_queue_msgid)
        self.table_queue.add_column(u'username', self.ksc_queue_username)
        self.table_queue.add_column(u'msgtext', self.ksc_queue_msgtext)
        self.table_queue.set_primary_column(u'msgid')

        self.ksem_event_manager = KSEventManager(self.uuid_number)
        self.ksutil = KSUtility(self.ksem_event_manager,
                                self.table_queue.get_transaction)

        with self.ksutil.transaction(self.__kstx_transaction, self.__init__) \
            as kstx_transaction:

            # Open the files
            self.table_public.open(0, 0660, kstx_transaction)
            self.table_private.open(0, 0660, kstx_transaction)
            self.table_queue.open(0, 0660, kstx_transaction)

            # Get messages from last session
            self.list_tw_pending_messages = \
                self.table_queue.keys(kstx_transaction)

            print self.list_tw_pending_messages
            self.table_queue.print_all(kstx_transaction)


        self.twitterapi.PostUpdate('Twitterbase is now running.')
        print 'Twitterbase is now running.'


    def user_exit(self):
        print 'Twitterbase is shutting down now.'
        self.twitterapi.PostUpdate('Twitterbase has shut down.')
        self.table_queue.close()
        self.table_public.close()
        self.table_private.close()
        sys.exit(0)

    def __del__(self):
        self.user_exit()


    def process(self):
        """
        Process messages from Twitter.

        """
        try:
            # Queue latest messages + delete messages from Twitter
            # Process a message from the queue and send the output
            while (True):
                time.sleep(30)
                self.print_all()
                print time.asctime()

                # Get new messages
                list_messages = self.twitterapi.GetDirectMessages()
                list_messages.reverse()
                print 'direct messages = ', list_messages

                # Queue the messages
                with self.ksutil.transaction(self.__kstx_transaction, self.process) \
                    as kstx_transaction:

                    try:
                        # Do a checkpoint if necessary
                        self.env.txn_checkpoint(0, 1000, 0)
                    except Exception:
                        print 'better luck next time'

                    for tw_message in list_messages:
                        # Check for halt command
                        if (tw_message.sender_screen_name == ''):
                            if (tw_message.text == ''):
                                self.twitterapi.DestroyDirectMessage(
                                    tw_message.id)
                                self.twitterapi.PostUpdate(
                                    'Twitterbase has shut down.')
                                print 'Twitterbase is shutting down now.'
                                sys.exit(0)

                        # Make sure that the message is not duplicated
                        list_results = self.table_queue.get_row_by_primary_value(
                            tw_message.id, [u'msgid'], kstx_transaction)

                        if (list_results == None):
                            # Message does not exist, queue it
                            obj_key = self.ksc_queue_msgid.generate_id()
                            self.table_queue.add_row(obj_key,
                                {u'msgid': tw_message.id,
                                 u'username': tw_message.sender_screen_name,
                                 u'msgtext': tw_message.text}, kstx_transaction)
                            self.list_tw_pending_messages.append(obj_key)

                            try:
                                self.twitterapi.DestroyDirectMessage(tw_message.id)
                            except HTTPError:
                                # figure that it went away, if not, it will
                                # re-queue and duplicate, and then go away
                                time.sleep(3)
                        else:
                            i_result_count = len(list_results)
                            self.twitterapi.DestroyDirectMessage(tw_message.id)
                            #if (i_result_count == 1):
                                # Message already exists, delete it
                            # else, ???

                # Get a new message + respond
                with self.ksutil.transaction(self.__kstx_transaction, self.process) \
                    as kstx_transaction:

                    if (len(self.list_tw_pending_messages) > 0):
                        # Get a new message
                        obj_key = self.list_tw_pending_messages[0]
                        list_msg = self.table_queue.get_row(
                            obj_key, [u'msgid', u'username', u'msgtext'],
                            kstx_transaction)

                        # Receive a single list with three items
                        if (len(list_msg)==3):
                            i_message_id = list_msg[0]
                            u_user_id = list_msg[1]
                            u_message_text = list_msg[2]

                            # Change the user for the session (used for security)
                            self.u_cmd_username = u_user_id

                            # Process a message (for now, just echo)
                            #u_output_text = 'Received: ' + u_message_text
                            u_output_text = self.external_parse(u_message_text)

                            # Send the message
                            try:
                                self.twitterapi.PostDirectMessage(
                                    u_user_id, u_output_text)

                                # Get awareness of data shared by others
                                if (self.b_display_public_message):
                                    u_status = u_message_text
                                    #u_status = '@%s: %s' % (
                                    #    self.u_cmd_username, u_message_text)
                                    self.twitterapi.PostUpdate(u_status)
                                # else, pass

                            except SystemError:
                                print 'outgoing problem'

                            # Delete from queue
                            self.table_queue.delete_row_by_primary_value(
                                i_message_id, kstx_transaction)
                            del self.list_tw_pending_messages[0]
                        else:
                            KSTwitterbaseError.print_and_raise_error(u'An error '
                                + u'occurred when retrieving the queued item.',
                                KSTwitterbaseError.B_CLASS_EXCEPTION)

                    # else, skip
            # end loop
        except KeyboardInterrupt:
            self.user_exit()

    def __xpath_cleaning(self, u_xpath):
        """
        Clean out XPath incosistencies. Use XPather for Firefox to get the
        best results.

        """
        try:
            if (isinstance(u_xpath, unicode)):
                # Clear out 'tbody' references
                no_tbody = re.compile(r'/tbody')
                u_xpath = no_tbody.sub('', u_xpath)

                correction = re.compile(r'\'')
                u_xpath = correction.sub('"', u_xpath)

                return u_xpath

            else:
                KSTwitterbaseError.print_and_raise_error(u'The XPath command '
                    + u'must be a unicode string.',
                    KSTwitterbaseError.B_CLASS_EXCEPTION)

        except Exception:
            KSTwitterbaseError.print_and_raise_error(u'The XPath command '
                + u'cleaner failed for an unknown reason.',
                KSTwitterbaseError.B_UNKNOWN_EXCEPTION)


    def xpath_processing(self, u_url_part, u_xpath_part):
        """
        Based on the URL and the XPath, return the value in that HTML or
        XML document.

        """
        try:
            if (not isinstance(u_url_part, unicode)):
                KSTwitterbaseError.print_and_raise_error(u'The URL '
                    + u'must be a unicode string.',
                    KSTwitterbaseError.B_CLASS_EXCEPTION)

            if (not isinstance(u_url_part, unicode)):
                KSTwitterbaseError.print_and_raise_error(u'The XPath '
                    + u'must be a unicode string.',
                    KSTwitterbaseError.B_CLASS_EXCEPTION)

            tree = etree.parse(u_url_part, self.parser)
            u_xpath_part = self.__xpath_cleaning(u_xpath_part)
            path = compile(u_xpath_part)

            result = path.evaluate(tree.getroot())
            for node in result:
                if ( node.tail != None ):
                    print node.text, node.tail
                else:
                    print node.text
            # end loop

        except Exception:
            KSTwitterbaseError.print_and_raise_error(u'An unknown error '
                + u'prevented the URL and XPath from being returned.',
                KSTwitterbaseError.B_UNKNOWN_EXCEPTION)


    def print_all(self):
        with self.ksutil.transaction(self.__kstx_transaction, self.print_all) \
            as kstx_transaction:

            self.table_public.print_all(kstx_transaction)
            self.table_private.print_all(kstx_transaction)
            self.table_queue.print_all(kstx_transaction)

    def external_parse(self, cmd):
        try:
            #print 'external parse'
            if ( cmd == u'`'):
                self.twitterapi.PostUpdate('Twitterbase has shut down.')
                print 'Twitterbase is shutting down now.'
                sys.exit(0)
                #self.user_exit()


            result = yacc.parse(cmd)
            try:
                #return cmd + ' :: ', unicode(result)
                if (result == None):
                    return 'Value = None; received: %s' % cmd
                else:
                    return unicode(result)
            except AttributeError:
                pass
        except LexerError, exp:
            exp.show()
        except ParserError, exp:
            exp.show()


class KSCommandShell(KSTwitterbase):
    USAGE_TEXT = """
    Usage:
        python python_201_parser_ply.py [options] <inputfile>
    Options:
        -h, --help      Display this help message.
    Example:
        python python_201_parser_ply.py testfile.prog
    """

    #def __init__(self, kscf_column_factory, kstf_transaction_factory):
    def __init__(self):
        super(KSCommandShell, self).__init__() #kscf_column_factory)
        #self.__kstf_transaction_factory = kstf_transaction_factory

    def build(self):
        # Build the lexer
        lex.lex(module=self, debug=1, optimize=0)
        # Build the parser
        yacc.yacc(module=self, debug=1, optimize=0)

    def usage(self):
        print KSCommandShell.USAGE_TEXT
        sys.exit(-1)

    def run_shell_parse(self):
        self.build()
        while 1:
            try:
                u_user_name = u''
                self.print_all()
                user_command = raw_input('(*) > ')
                user_command = unicode(user_command[:-1])
                self.external_parse(user_command)
            except LexerError, exp:
                exp.show()
            except ParserError, exp:
                exp.show()
            except KSTwitterbaseError:
                print 'cannot make directories for %s; exiting...' % u_user_name
                sys.exit(0)
            except Exception:
                continue
            except KeyboardInterrupt:
                sys.exit(0)

####################

try:
    kstw = KSCommandShell()
    kstw.build()
    kstw.process()
    #kstw.run_shell_parse()
except Exception:
    print 'shut down'